Create chips for the majority of GB¶
This notebook samples from the majority of GB and dumps them to disk as TIF files to be loaded by TensorFlow.
import glob
import os
import requests
import shutil
import warnings
from random import shuffle
from itertools import product, chain
from pathlib import Path
import contextily
import dask
import dask_geopandas
import dask.bag
import dask.dataframe
import geopandas
import numpy
import pandas
import pygeos
import pyogrio
import xarray, rioxarray
import libpysal
from dask_geopandas.hilbert_distance import _hilbert_distance
from dask.distributed import Client, LocalCluster
from shapely.geometry import box
from shapely.ops import polygonize
from tqdm.auto import tqdm
from sqlalchemy import create_engine
import tools
client = Client(
LocalCluster(n_workers=16, threads_per_worker=1)
)
client
Client
Client-f5b49697-8fdf-11ec-84d2-dfb21f7d7f99
| Connection method: Cluster object | Cluster type: distributed.LocalCluster |
| Dashboard: http://127.0.0.1:8787/status |
Cluster Info
LocalCluster
75cd7627
| Dashboard: http://127.0.0.1:8787/status | Workers: 16 |
| Total threads: 16 | Total memory: 125.54 GiB |
| Status: running | Using processes: True |
Scheduler Info
Scheduler
Scheduler-6f932a10-9fcf-42f4-8bba-c95b527cd647
| Comm: tcp://127.0.0.1:39263 | Workers: 16 |
| Dashboard: http://127.0.0.1:8787/status | Total threads: 16 |
| Started: Just now | Total memory: 125.54 GiB |
Workers
Worker: 0
| Comm: tcp://127.0.0.1:39163 | Total threads: 1 |
| Dashboard: http://127.0.0.1:35573/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:35749 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-jsttgpnr | |
Worker: 1
| Comm: tcp://127.0.0.1:34813 | Total threads: 1 |
| Dashboard: http://127.0.0.1:35433/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:39267 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-vxlduu4g | |
Worker: 2
| Comm: tcp://127.0.0.1:43269 | Total threads: 1 |
| Dashboard: http://127.0.0.1:39765/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:38731 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-x2aqt2ql | |
Worker: 3
| Comm: tcp://127.0.0.1:44671 | Total threads: 1 |
| Dashboard: http://127.0.0.1:46343/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:39539 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-thy4jzfm | |
Worker: 4
| Comm: tcp://127.0.0.1:45141 | Total threads: 1 |
| Dashboard: http://127.0.0.1:33085/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:34511 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-fotpiwta | |
Worker: 5
| Comm: tcp://127.0.0.1:35105 | Total threads: 1 |
| Dashboard: http://127.0.0.1:33581/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:44093 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-tbjs6md1 | |
Worker: 6
| Comm: tcp://127.0.0.1:33231 | Total threads: 1 |
| Dashboard: http://127.0.0.1:46771/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:37917 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-xzz1f6l8 | |
Worker: 7
| Comm: tcp://127.0.0.1:36701 | Total threads: 1 |
| Dashboard: http://127.0.0.1:44283/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:43849 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-wm69fi4n | |
Worker: 8
| Comm: tcp://127.0.0.1:33965 | Total threads: 1 |
| Dashboard: http://127.0.0.1:35493/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:42359 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-qstyj2kz | |
Worker: 9
| Comm: tcp://127.0.0.1:36481 | Total threads: 1 |
| Dashboard: http://127.0.0.1:34549/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:33849 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-d3pyo5ue | |
Worker: 10
| Comm: tcp://127.0.0.1:40773 | Total threads: 1 |
| Dashboard: http://127.0.0.1:45539/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:46115 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-jzexfya0 | |
Worker: 11
| Comm: tcp://127.0.0.1:34931 | Total threads: 1 |
| Dashboard: http://127.0.0.1:40253/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:37135 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-u7uqavok | |
Worker: 12
| Comm: tcp://127.0.0.1:40877 | Total threads: 1 |
| Dashboard: http://127.0.0.1:41057/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:41913 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-5hezvlr2 | |
Worker: 13
| Comm: tcp://127.0.0.1:38169 | Total threads: 1 |
| Dashboard: http://127.0.0.1:34835/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:33809 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-9am0njdf | |
Worker: 14
| Comm: tcp://127.0.0.1:45719 | Total threads: 1 |
| Dashboard: http://127.0.0.1:40865/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:33955 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-5i_slymv | |
Worker: 15
| Comm: tcp://127.0.0.1:41477 | Total threads: 1 |
| Dashboard: http://127.0.0.1:35023/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:43627 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-60q3eqyk | |
Specs¶
specs = {
'chip_size': 32,
'bands': [1, 2, 3], #RGB
'mosaic_p': (
'/home/jovyan/work/urbangrammar_samba/'
'ghs_composite_s2/GHS-composite-S2.vrt'
),
'spsig_p': (
'/home/jovyan/work/urbangrammar_samba/spatial_signatures/'
'signatures/'
'signatures_combined_levels_simplified_clip.gpkg'
),
'points_temp': '/home/jovyan/work/chips_gb/temp/points/',
'folder': (
'/home/jovyan/work/chips_gb/32_shuffled/'
),
}
Load signatures and Sentinel2 mosaic¶
%%time
spsig = pyogrio.read_dataframe(specs['spsig_p'])
CPU times: user 848 ms, sys: 355 ms, total: 1.2 s
Wall time: 3.48 s
mosaic = rioxarray.open_rasterio(
specs['mosaic_p'], chunks={'x': 1024, 'y': 1024}
)
Get range of coordinates from the mosaic¶
start_x = float(mosaic["x"].min())
start_y = float(mosaic["y"].min())
end_x = float(mosaic["x"].max())
end_y = float(mosaic["y"].max())
Create coordinates of individual points at set chip size. We are using only the southern half of the extent that covers the country up to Perth.¶
x_coords = numpy.arange(start_x, end_x, specs["chip_size"] * 10)
y_coords = numpy.arange(start_y, end_y, specs["chip_size"] * 10)
x_bag = dask.bag.from_sequence(x_coords)
y_bag = dask.bag.from_sequence(y_coords[:y_coords.shape[0]//2])
product_bag = x_bag.product(y_bag)
ddf = product_bag.to_dataframe(meta={"x": float, "y":float})
%%time
ddf.to_parquet("/home/jovyan/work/chips_gb/temp/coords/", overwrite=True)
CPU times: user 14.7 s, sys: 836 ms, total: 15.5 s
Wall time: 17.2 s
[None]
Create points from coordinates¶
ddf = dask.dataframe.read_parquet("/home/jovyan/work/chips_gb/temp/coords/")
ddf["geometry"] = dask_geopandas.points_from_xy(ddf, "x", "y", crs=27700)
gddf = dask_geopandas.from_dask_dataframe(ddf).set_crs(27700)
gddf.to_parquet(specs["points_temp"], overwrite=True)
client.restart()
Client
Client-a8ea1a6b-8fdd-11ec-80c7-ad6fa5f97e07
| Connection method: Cluster object | Cluster type: distributed.LocalCluster |
| Dashboard: http://127.0.0.1:8787/status |
Cluster Info
LocalCluster
2c62e670
| Dashboard: http://127.0.0.1:8787/status | Workers: 16 |
| Total threads: 16 | Total memory: 125.54 GiB |
| Status: running | Using processes: True |
Scheduler Info
Scheduler
Scheduler-d0d30d39-92dc-42c1-8952-46c41aabb58d
| Comm: tcp://127.0.0.1:33859 | Workers: 16 |
| Dashboard: http://127.0.0.1:8787/status | Total threads: 16 |
| Started: 5 minutes ago | Total memory: 125.54 GiB |
Workers
Worker: 0
| Comm: tcp://127.0.0.1:34433 | Total threads: 1 |
| Dashboard: http://127.0.0.1:39583/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:44499 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-a7assn_n | |
Worker: 1
| Comm: tcp://127.0.0.1:44659 | Total threads: 1 |
| Dashboard: http://127.0.0.1:37565/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:33689 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-fxlgk7zt | |
Worker: 2
| Comm: tcp://127.0.0.1:46855 | Total threads: 1 |
| Dashboard: http://127.0.0.1:41585/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:33477 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-xaobh09c | |
Worker: 3
| Comm: tcp://127.0.0.1:45521 | Total threads: 1 |
| Dashboard: http://127.0.0.1:38359/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:45833 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-egu_i698 | |
Worker: 4
| Comm: tcp://127.0.0.1:36529 | Total threads: 1 |
| Dashboard: http://127.0.0.1:37083/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:42035 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-7ctt8cjs | |
Worker: 5
| Comm: tcp://127.0.0.1:41033 | Total threads: 1 |
| Dashboard: http://127.0.0.1:45487/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:33363 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-cxkfhjor | |
Worker: 6
| Comm: tcp://127.0.0.1:39057 | Total threads: 1 |
| Dashboard: http://127.0.0.1:37137/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:42919 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-vix5fjvz | |
Worker: 7
| Comm: tcp://127.0.0.1:35901 | Total threads: 1 |
| Dashboard: http://127.0.0.1:41655/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:44255 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-2m3i_mmh | |
Worker: 8
| Comm: tcp://127.0.0.1:45803 | Total threads: 1 |
| Dashboard: http://127.0.0.1:40783/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:34057 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-694o4adm | |
Worker: 9
| Comm: tcp://127.0.0.1:33993 | Total threads: 1 |
| Dashboard: http://127.0.0.1:45763/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:43861 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-s_j8daks | |
Worker: 10
| Comm: tcp://127.0.0.1:39365 | Total threads: 1 |
| Dashboard: http://127.0.0.1:32817/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:43763 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-28ky0q9d | |
Worker: 11
| Comm: tcp://127.0.0.1:36809 | Total threads: 1 |
| Dashboard: http://127.0.0.1:34571/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:43165 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-w9wkodin | |
Worker: 12
| Comm: tcp://127.0.0.1:46457 | Total threads: 1 |
| Dashboard: http://127.0.0.1:39261/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:39837 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-65gglzn2 | |
Worker: 13
| Comm: tcp://127.0.0.1:45827 | Total threads: 1 |
| Dashboard: http://127.0.0.1:41301/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:36225 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-qs23pka9 | |
Worker: 14
| Comm: tcp://127.0.0.1:37649 | Total threads: 1 |
| Dashboard: http://127.0.0.1:42449/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:36473 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-zthg75aa | |
Worker: 15
| Comm: tcp://127.0.0.1:40187 | Total threads: 1 |
| Dashboard: http://127.0.0.1:38157/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:45155 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-mi1s2mdi | |
Filter points that fall on land.¶
Clip by the coastline
Spatial join
signatures = spsig[['signature_type', 'geometry']].set_index(_hilbert_distance(spsig, spsig.total_bounds, level=10))
points = dask_geopandas.read_parquet(specs["points_temp"])
points_within = dask_geopandas.sjoin(points, signatures, how="inner", op='within')
/tmp/ipykernel_199/3757199691.py:1: FutureWarning: The `op` parameter is deprecated and will be removed in a future release. Please use the `predicate` parameter instead.
points_within = dask_geopandas.sjoin(points, signatures, how="inner", op='within')
points_within.to_parquet("/home/jovyan/work/chips_gb/temp/points_within/", overwrite=True)
client.restart()
Client
Client-a8ea1a6b-8fdd-11ec-80c7-ad6fa5f97e07
| Connection method: Cluster object | Cluster type: distributed.LocalCluster |
| Dashboard: http://127.0.0.1:8787/status |
Cluster Info
LocalCluster
2c62e670
| Dashboard: http://127.0.0.1:8787/status | Workers: 16 |
| Total threads: 16 | Total memory: 125.54 GiB |
| Status: running | Using processes: True |
Scheduler Info
Scheduler
Scheduler-d0d30d39-92dc-42c1-8952-46c41aabb58d
| Comm: tcp://127.0.0.1:33859 | Workers: 16 |
| Dashboard: http://127.0.0.1:8787/status | Total threads: 16 |
| Started: 11 minutes ago | Total memory: 125.54 GiB |
Workers
Worker: 0
| Comm: tcp://127.0.0.1:40897 | Total threads: 1 |
| Dashboard: http://127.0.0.1:39805/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:44499 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-f2wn9gc_ | |
Worker: 1
| Comm: tcp://127.0.0.1:40757 | Total threads: 1 |
| Dashboard: http://127.0.0.1:37179/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:33689 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-y24_iuzs | |
Worker: 2
| Comm: tcp://127.0.0.1:42967 | Total threads: 1 |
| Dashboard: http://127.0.0.1:44785/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:33477 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-5i2e3u8t | |
Worker: 3
| Comm: tcp://127.0.0.1:45443 | Total threads: 1 |
| Dashboard: http://127.0.0.1:46081/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:45833 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-3xosaylj | |
Worker: 4
| Comm: tcp://127.0.0.1:45183 | Total threads: 1 |
| Dashboard: http://127.0.0.1:33771/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:42035 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-va30xjua | |
Worker: 5
| Comm: tcp://127.0.0.1:44083 | Total threads: 1 |
| Dashboard: http://127.0.0.1:38925/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:33363 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-36aqk78b | |
Worker: 6
| Comm: tcp://127.0.0.1:39221 | Total threads: 1 |
| Dashboard: http://127.0.0.1:38491/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:42919 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-4ouvw79l | |
Worker: 7
| Comm: tcp://127.0.0.1:46181 | Total threads: 1 |
| Dashboard: http://127.0.0.1:40713/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:44255 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-wlvcn521 | |
Worker: 8
| Comm: tcp://127.0.0.1:34851 | Total threads: 1 |
| Dashboard: http://127.0.0.1:32805/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:34057 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-k5097voa | |
Worker: 9
| Comm: tcp://127.0.0.1:34161 | Total threads: 1 |
| Dashboard: http://127.0.0.1:46529/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:43861 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-d6cd5gb6 | |
Worker: 10
| Comm: tcp://127.0.0.1:36095 | Total threads: 1 |
| Dashboard: http://127.0.0.1:33215/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:43763 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-9h4x5abm | |
Worker: 11
| Comm: tcp://127.0.0.1:37795 | Total threads: 1 |
| Dashboard: http://127.0.0.1:36905/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:43165 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-_pn24jrx | |
Worker: 12
| Comm: tcp://127.0.0.1:46473 | Total threads: 1 |
| Dashboard: http://127.0.0.1:34745/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:39837 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-92i8n9d5 | |
Worker: 13
| Comm: tcp://127.0.0.1:39043 | Total threads: 1 |
| Dashboard: http://127.0.0.1:44727/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:36225 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-x8lky7n3 | |
Worker: 14
| Comm: tcp://127.0.0.1:45749 | Total threads: 1 |
| Dashboard: http://127.0.0.1:37035/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:36473 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-4jpulknp | |
Worker: 15
| Comm: tcp://127.0.0.1:45255 | Total threads: 1 |
| Dashboard: http://127.0.0.1:37125/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:45155 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-6ouqtkn1 | |
Create chip bounds polygons.¶
points_within = dask.dataframe.read_parquet("/home/jovyan/work/chips_gb/temp/points_within/", columns=["geometry"])
points_within["geometry"] = points_within["geometry"].map_partitions(geopandas.GeoSeries.from_wkb, meta=geopandas.GeoSeries())
points_within = dask_geopandas.from_dask_dataframe(points_within)
polygons = points_within.buffer(specs['chip_size'] * 10 / 2, cap_style=3)
points_within["geometry"] = polygons
points_within = points_within.repartition(partition_size="1MB")
points_within.to_parquet("/home/jovyan/work/chips_gb/temp/polygons/", overwrite=True)
Dump signatures to sorted partitioned parquet.¶
signatures = spsig[['signature_type', 'geometry']].set_index(_hilbert_distance(spsig, spsig.total_bounds, level=10))
signatures = dask_geopandas.from_geopandas(signatures.sort_index(), npartitions=1000)
signatures.to_parquet("/home/jovyan/work/chips_gb/temp/signatures")
---------------------------------------------------------------------------
KeyboardInterrupt Traceback (most recent call last)
/tmp/ipykernel_199/798855391.py in <module>
1 signatures = dask_geopandas.from_geopandas(signatures.sort_index(), npartitions=1000)
----> 2 signatures.to_parquet("/home/jovyan/work/chips_gb/temp/signatures")
/opt/conda/lib/python3.9/site-packages/dask_geopandas/core.py in to_parquet(self, path, *args, **kwargs)
585 from .io.parquet import to_parquet
586
--> 587 return to_parquet(self, path, *args, **kwargs)
588
589 def to_feather(self, path, *args, **kwargs):
/opt/conda/lib/python3.9/site-packages/dask/dataframe/io/parquet/core.py in to_parquet(df, path, engine, compression, write_index, append, overwrite, ignore_divisions, partition_on, storage_options, custom_metadata, write_metadata_file, compute, compute_kwargs, schema, name_function, **kwargs)
841 graph = HighLevelGraph.from_collections(meta_name, dsk, dependencies=(data_write,))
842 if compute:
--> 843 return compute_as_if_collection(
844 Scalar, graph, [(meta_name, 0)], **compute_kwargs
845 )
/opt/conda/lib/python3.9/site-packages/dask/base.py in compute_as_if_collection(cls, dsk, keys, scheduler, get, **kwargs)
315 schedule = get_scheduler(scheduler=scheduler, cls=cls, get=get)
316 dsk2 = optimization_function(cls)(dsk, keys, **kwargs)
--> 317 return schedule(dsk2, keys, **kwargs)
318
319
/opt/conda/lib/python3.9/site-packages/distributed/client.py in get(self, dsk, keys, workers, allow_other_workers, resources, sync, asynchronous, direct, retries, priority, fifo_timeout, actors, **kwargs)
2992 should_rejoin = False
2993 try:
-> 2994 results = self.gather(packed, asynchronous=asynchronous, direct=direct)
2995 finally:
2996 for f in futures.values():
/opt/conda/lib/python3.9/site-packages/distributed/client.py in gather(self, futures, errors, direct, asynchronous)
2144 else:
2145 local_worker = None
-> 2146 return self.sync(
2147 self._gather,
2148 futures,
/opt/conda/lib/python3.9/site-packages/distributed/utils.py in sync(self, func, asynchronous, callback_timeout, *args, **kwargs)
307 return future
308 else:
--> 309 return sync(
310 self.loop, func, *args, callback_timeout=callback_timeout, **kwargs
311 )
/opt/conda/lib/python3.9/site-packages/distributed/utils.py in sync(loop, func, callback_timeout, *args, **kwargs)
370 else:
371 while not e.is_set():
--> 372 wait(10)
373
374 if error:
/opt/conda/lib/python3.9/site-packages/distributed/utils.py in wait(timeout)
359 def wait(timeout):
360 try:
--> 361 return e.wait(timeout)
362 except KeyboardInterrupt:
363 loop.add_callback(cancel)
/opt/conda/lib/python3.9/threading.py in wait(self, timeout)
572 signaled = self._flag
573 if not signaled:
--> 574 signaled = self._cond.wait(timeout)
575 return signaled
576
/opt/conda/lib/python3.9/threading.py in wait(self, timeout)
314 else:
315 if timeout > 0:
--> 316 gotit = waiter.acquire(True, timeout)
317 else:
318 gotit = waiter.acquire(False)
KeyboardInterrupt:
client.restart()
Client
Client-7babfc26-7df9-11ec-a48a-f5d3c2b9d425
| Connection method: Cluster object | Cluster type: distributed.LocalCluster |
| Dashboard: http://127.0.0.1:8787/status |
Cluster Info
LocalCluster
13b21a60
| Dashboard: http://127.0.0.1:8787/status | Workers: 16 |
| Total threads: 16 | Total memory: 125.54 GiB |
| Status: running | Using processes: True |
Scheduler Info
Scheduler
Scheduler-e81b277a-ada7-4507-8559-2291801af378
| Comm: tcp://127.0.0.1:39999 | Workers: 16 |
| Dashboard: http://127.0.0.1:8787/status | Total threads: 16 |
| Started: 6 minutes ago | Total memory: 125.54 GiB |
Workers
Worker: 0
| Comm: tcp://172.17.0.3:38841 | Total threads: 1 |
| Dashboard: http://172.17.0.3:35885/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:45457 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-fzmge5xj | |
Worker: 1
| Comm: tcp://172.17.0.3:39713 | Total threads: 1 |
| Dashboard: http://172.17.0.3:33465/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:33069 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-o_zg77ay | |
Worker: 2
| Comm: tcp://172.17.0.3:46537 | Total threads: 1 |
| Dashboard: http://172.17.0.3:35545/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:40239 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-842kcren | |
Worker: 3
| Comm: tcp://172.17.0.3:45369 | Total threads: 1 |
| Dashboard: http://172.17.0.3:42065/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:45875 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-bk0shk_r | |
Worker: 4
| Comm: tcp://172.17.0.3:41419 | Total threads: 1 |
| Dashboard: http://172.17.0.3:37917/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:40047 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-p2ews4wd | |
Worker: 5
| Comm: tcp://172.17.0.3:34543 | Total threads: 1 |
| Dashboard: http://172.17.0.3:40159/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:37215 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-_p5d0h5c | |
Worker: 6
| Comm: tcp://172.17.0.3:42625 | Total threads: 1 |
| Dashboard: http://172.17.0.3:39831/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:34967 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-2k6a9w3i | |
Worker: 7
| Comm: tcp://172.17.0.3:39995 | Total threads: 1 |
| Dashboard: http://172.17.0.3:40555/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:34367 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-nra0v3vs | |
Worker: 8
| Comm: tcp://172.17.0.3:43623 | Total threads: 1 |
| Dashboard: http://172.17.0.3:41281/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:45873 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-7ist2nl8 | |
Worker: 9
| Comm: tcp://172.17.0.3:33337 | Total threads: 1 |
| Dashboard: http://172.17.0.3:43031/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:33331 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-g4amv753 | |
Worker: 10
| Comm: tcp://172.17.0.3:36379 | Total threads: 1 |
| Dashboard: http://172.17.0.3:41031/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:40771 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-4nkda4e2 | |
Worker: 11
| Comm: tcp://172.17.0.3:33811 | Total threads: 1 |
| Dashboard: http://172.17.0.3:45317/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:37103 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-ze6nfo0n | |
Worker: 12
| Comm: tcp://172.17.0.3:45691 | Total threads: 1 |
| Dashboard: http://172.17.0.3:33261/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:44359 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-7726re82 | |
Worker: 13
| Comm: tcp://172.17.0.3:33457 | Total threads: 1 |
| Dashboard: http://172.17.0.3:43593/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:41327 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-ldei1qv_ | |
Worker: 14
| Comm: tcp://172.17.0.3:37053 | Total threads: 1 |
| Dashboard: http://172.17.0.3:40875/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:39477 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-7jsj1slu | |
Worker: 15
| Comm: tcp://172.17.0.3:42761 | Total threads: 1 |
| Dashboard: http://172.17.0.3:37005/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:35315 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-0r7zq99c | |
Filter polygons fully wihtin a single signature type.¶
polygons = dask_geopandas.read_parquet("/home/jovyan/work/chips_gb/temp/polygons/").set_crs(27700)
polygons_within = dask_geopandas.sjoin(polygons, spsig[['signature_type', 'geometry']], op="within")
/tmp/ipykernel_1234/3882422153.py:1: FutureWarning: The `op` parameter is deprecated and will be removed in a future release. Please use the `predicate` parameter instead.
polygons_within = dask_geopandas.sjoin(polygons, spsig[['signature_type', 'geometry']], op="within")
polygons_within.to_parquet("/home/jovyan/work/chips_gb/temp/chip_bounds/", overwrite=True)
/opt/conda/lib/python3.9/site-packages/dask/dataframe/io/parquet/core.py:157: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
return self.engine.write_partition(
/opt/conda/lib/python3.9/site-packages/dask/dataframe/io/parquet/core.py:157: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
return self.engine.write_partition(
/opt/conda/lib/python3.9/site-packages/dask/dataframe/io/parquet/core.py:157: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
return self.engine.write_partition(
/opt/conda/lib/python3.9/site-packages/dask/dataframe/io/parquet/core.py:157: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
return self.engine.write_partition(
/opt/conda/lib/python3.9/site-packages/dask/dataframe/io/parquet/core.py:157: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
return self.engine.write_partition(
/opt/conda/lib/python3.9/site-packages/dask/dataframe/io/parquet/core.py:157: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
return self.engine.write_partition(
/opt/conda/lib/python3.9/site-packages/dask/dataframe/io/parquet/core.py:157: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
return self.engine.write_partition(
/opt/conda/lib/python3.9/site-packages/dask/dataframe/io/parquet/core.py:157: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
return self.engine.write_partition(
/opt/conda/lib/python3.9/site-packages/dask/dataframe/io/parquet/core.py:157: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
return self.engine.write_partition(
/opt/conda/lib/python3.9/site-packages/dask/dataframe/io/parquet/core.py:157: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
return self.engine.write_partition(
/opt/conda/lib/python3.9/site-packages/dask/dataframe/io/parquet/core.py:157: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
return self.engine.write_partition(
/opt/conda/lib/python3.9/site-packages/dask/dataframe/io/parquet/core.py:157: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
return self.engine.write_partition(
/opt/conda/lib/python3.9/site-packages/dask/dataframe/io/parquet/core.py:157: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
return self.engine.write_partition(
/opt/conda/lib/python3.9/site-packages/dask/dataframe/io/parquet/core.py:157: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
return self.engine.write_partition(
/opt/conda/lib/python3.9/site-packages/dask/dataframe/io/parquet/core.py:157: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
return self.engine.write_partition(
/opt/conda/lib/python3.9/site-packages/dask/dataframe/io/parquet/core.py:157: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
return self.engine.write_partition(
/opt/conda/lib/python3.9/site-packages/dask/dataframe/io/parquet/core.py:157: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
return self.engine.write_partition(
/opt/conda/lib/python3.9/site-packages/dask/dataframe/io/parquet/core.py:157: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
return self.engine.write_partition(
/opt/conda/lib/python3.9/site-packages/dask/dataframe/io/parquet/core.py:157: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
return self.engine.write_partition(
/opt/conda/lib/python3.9/site-packages/dask/dataframe/io/parquet/core.py:157: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
return self.engine.write_partition(
/opt/conda/lib/python3.9/site-packages/dask/dataframe/io/parquet/core.py:157: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
return self.engine.write_partition(
/opt/conda/lib/python3.9/site-packages/dask/dataframe/io/parquet/core.py:157: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
return self.engine.write_partition(
/opt/conda/lib/python3.9/site-packages/dask/dataframe/io/parquet/core.py:157: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
return self.engine.write_partition(
/opt/conda/lib/python3.9/site-packages/dask/dataframe/io/parquet/core.py:157: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
return self.engine.write_partition(
/opt/conda/lib/python3.9/site-packages/dask/dataframe/io/parquet/core.py:157: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
return self.engine.write_partition(
/opt/conda/lib/python3.9/site-packages/dask/dataframe/io/parquet/core.py:157: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
return self.engine.write_partition(
/opt/conda/lib/python3.9/site-packages/dask/dataframe/io/parquet/core.py:157: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
return self.engine.write_partition(
/opt/conda/lib/python3.9/site-packages/dask/dataframe/io/parquet/core.py:157: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
return self.engine.write_partition(
[None]
client.restart()
Client
Client-f5b49697-8fdf-11ec-84d2-dfb21f7d7f99
| Connection method: Cluster object | Cluster type: distributed.LocalCluster |
| Dashboard: http://127.0.0.1:8787/status |
Cluster Info
LocalCluster
75cd7627
| Dashboard: http://127.0.0.1:8787/status | Workers: 16 |
| Total threads: 16 | Total memory: 125.54 GiB |
| Status: running | Using processes: True |
Scheduler Info
Scheduler
Scheduler-6f932a10-9fcf-42f4-8bba-c95b527cd647
| Comm: tcp://127.0.0.1:39263 | Workers: 16 |
| Dashboard: http://127.0.0.1:8787/status | Total threads: 16 |
| Started: 1 minute ago | Total memory: 125.54 GiB |
Workers
Worker: 0
| Comm: tcp://127.0.0.1:44153 | Total threads: 1 |
| Dashboard: http://127.0.0.1:39183/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:35749 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-cgxf2c76 | |
Worker: 1
| Comm: tcp://127.0.0.1:41819 | Total threads: 1 |
| Dashboard: http://127.0.0.1:39767/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:39267 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-kdon2u4t | |
Worker: 2
| Comm: tcp://127.0.0.1:37523 | Total threads: 1 |
| Dashboard: http://127.0.0.1:44197/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:38731 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-rznkqnem | |
Worker: 3
| Comm: tcp://127.0.0.1:36061 | Total threads: 1 |
| Dashboard: http://127.0.0.1:38133/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:39539 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-kziciabo | |
Worker: 4
| Comm: tcp://127.0.0.1:33787 | Total threads: 1 |
| Dashboard: http://127.0.0.1:33433/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:34511 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-reqoocha | |
Worker: 5
| Comm: tcp://127.0.0.1:40073 | Total threads: 1 |
| Dashboard: http://127.0.0.1:41443/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:44093 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-1vgqfja9 | |
Worker: 6
| Comm: tcp://127.0.0.1:34879 | Total threads: 1 |
| Dashboard: http://127.0.0.1:37249/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:37917 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-id6au09q | |
Worker: 7
| Comm: tcp://127.0.0.1:34591 | Total threads: 1 |
| Dashboard: http://127.0.0.1:41729/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:43849 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-dszf0l7d | |
Worker: 8
| Comm: tcp://127.0.0.1:45091 | Total threads: 1 |
| Dashboard: http://127.0.0.1:37091/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:42359 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-kldwy0o5 | |
Worker: 9
| Comm: tcp://127.0.0.1:40717 | Total threads: 1 |
| Dashboard: http://127.0.0.1:34893/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:33849 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-czj_66qs | |
Worker: 10
| Comm: tcp://127.0.0.1:42071 | Total threads: 1 |
| Dashboard: http://127.0.0.1:37351/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:46115 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-tyqh5y58 | |
Worker: 11
| Comm: tcp://127.0.0.1:36947 | Total threads: 1 |
| Dashboard: http://127.0.0.1:39097/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:37135 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-7ely6laz | |
Worker: 12
| Comm: tcp://127.0.0.1:35003 | Total threads: 1 |
| Dashboard: http://127.0.0.1:43931/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:41913 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-dzuctk8b | |
Worker: 13
| Comm: tcp://127.0.0.1:39737 | Total threads: 1 |
| Dashboard: http://127.0.0.1:36971/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:33809 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-31xf1r5m | |
Worker: 14
| Comm: tcp://127.0.0.1:46193 | Total threads: 1 |
| Dashboard: http://127.0.0.1:37985/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:33955 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-lroo35da | |
Worker: 15
| Comm: tcp://127.0.0.1:37087 | Total threads: 1 |
| Dashboard: http://127.0.0.1:37575/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:43627 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-bxa99j2l | |
How many chips we have per type?
polygons_within = dask_geopandas.read_parquet("/home/jovyan/work/chips_gb/temp/chip_bounds/")
val_counts = polygons_within["signature_type"].value_counts().compute()
val_counts
0_0 766870
4_0 408232
7_0 206819
3_0 16205
5_0 10031
1_0 7241
2_0 1757
2_2 1716
2_1 1103
8_0 1080
9_0 812
6_0 736
9_2 272
9_4 46
9_1 30
9_5 5
9_6 1
Name: signature_type, dtype: int64
If you want to do sliding based augmentation, skip the following steps to the Augmentation part.
Get sample of max 50k chips per type.¶
chip_bds = dask_geopandas.read_parquet("/home/jovyan/work/chips_gb/temp/chip_bounds/", columns=["geometry", "signature_type"])
types = ['0_0', '1_0', '3_0', '4_0', '5_0', '6_0', '7_0', '8_0', '2_0', '2_1', '2_2', '9_0', '9_1', '9_2', '9_4', '9_5']
for t in types:
df = chip_bds[chip_bds["signature_type"] == t].compute()
n = 50_000 if len(df) > 50_000 else len(df)
df.sample(n=n, random_state=42).to_parquet(f"/home/jovyan/work/chips_gb/temp/chip_bounds_per_type/{t}.parquet")
print(t)
/tmp/ipykernel_65930/477179734.py:4: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
df.sample(n=n, random_state=42).to_parquet(f"/home/jovyan/work/chips_gb/temp/chip_bounds_per_type/{t}.parquet")
0_0
/tmp/ipykernel_65930/477179734.py:4: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
df.sample(n=n, random_state=42).to_parquet(f"/home/jovyan/work/chips_gb/temp/chip_bounds_per_type/{t}.parquet")
1_0
/tmp/ipykernel_65930/477179734.py:4: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
df.sample(n=n, random_state=42).to_parquet(f"/home/jovyan/work/chips_gb/temp/chip_bounds_per_type/{t}.parquet")
3_0
/tmp/ipykernel_65930/477179734.py:4: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
df.sample(n=n, random_state=42).to_parquet(f"/home/jovyan/work/chips_gb/temp/chip_bounds_per_type/{t}.parquet")
4_0
/tmp/ipykernel_65930/477179734.py:4: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
df.sample(n=n, random_state=42).to_parquet(f"/home/jovyan/work/chips_gb/temp/chip_bounds_per_type/{t}.parquet")
5_0
/tmp/ipykernel_65930/477179734.py:4: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
df.sample(n=n, random_state=42).to_parquet(f"/home/jovyan/work/chips_gb/temp/chip_bounds_per_type/{t}.parquet")
6_0
/tmp/ipykernel_65930/477179734.py:4: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
df.sample(n=n, random_state=42).to_parquet(f"/home/jovyan/work/chips_gb/temp/chip_bounds_per_type/{t}.parquet")
7_0
/tmp/ipykernel_65930/477179734.py:4: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
df.sample(n=n, random_state=42).to_parquet(f"/home/jovyan/work/chips_gb/temp/chip_bounds_per_type/{t}.parquet")
8_0
/tmp/ipykernel_65930/477179734.py:4: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
df.sample(n=n, random_state=42).to_parquet(f"/home/jovyan/work/chips_gb/temp/chip_bounds_per_type/{t}.parquet")
2_0
/tmp/ipykernel_65930/477179734.py:4: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
df.sample(n=n, random_state=42).to_parquet(f"/home/jovyan/work/chips_gb/temp/chip_bounds_per_type/{t}.parquet")
2_1
/tmp/ipykernel_65930/477179734.py:4: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
df.sample(n=n, random_state=42).to_parquet(f"/home/jovyan/work/chips_gb/temp/chip_bounds_per_type/{t}.parquet")
2_2
/tmp/ipykernel_65930/477179734.py:4: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
df.sample(n=n, random_state=42).to_parquet(f"/home/jovyan/work/chips_gb/temp/chip_bounds_per_type/{t}.parquet")
9_0
/tmp/ipykernel_65930/477179734.py:4: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
df.sample(n=n, random_state=42).to_parquet(f"/home/jovyan/work/chips_gb/temp/chip_bounds_per_type/{t}.parquet")
9_1
/tmp/ipykernel_65930/477179734.py:4: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
df.sample(n=n, random_state=42).to_parquet(f"/home/jovyan/work/chips_gb/temp/chip_bounds_per_type/{t}.parquet")
9_2
/tmp/ipykernel_65930/477179734.py:4: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
df.sample(n=n, random_state=42).to_parquet(f"/home/jovyan/work/chips_gb/temp/chip_bounds_per_type/{t}.parquet")
9_4
9_5
/tmp/ipykernel_65930/477179734.py:4: UserWarning: this is an initial implementation of Parquet/Feather file support and associated metadata. This is tracking version 0.1.0 of the metadata specification at https://github.com/geopandas/geo-arrow-spec
This metadata specification does not yet make stability promises. We do not yet recommend using this in a production setting unless you are able to rewrite your Parquet/Feather files.
To further ignore this warning, you can do:
import warnings; warnings.filterwarnings('ignore', message='.*initial implementation of Parquet.*')
df.sample(n=n, random_state=42).to_parquet(f"/home/jovyan/work/chips_gb/temp/chip_bounds_per_type/{t}.parquet")
client.restart()
Client
Client-a865dce7-84e9-11ec-818a-634439067d6f
| Connection method: Cluster object | Cluster type: distributed.LocalCluster |
| Dashboard: http://127.0.0.1:8787/status |
Cluster Info
LocalCluster
eba53ebb
| Dashboard: http://127.0.0.1:8787/status | Workers: 16 |
| Total threads: 16 | Total memory: 125.54 GiB |
| Status: running | Using processes: True |
Scheduler Info
Scheduler
Scheduler-0d7447ec-f43e-41a9-ab1c-f74a7bb6d7ca
| Comm: tcp://127.0.0.1:46427 | Workers: 16 |
| Dashboard: http://127.0.0.1:8787/status | Total threads: 16 |
| Started: 7 minutes ago | Total memory: 125.54 GiB |
Workers
Worker: 0
| Comm: tcp://172.17.0.3:32961 | Total threads: 1 |
| Dashboard: http://172.17.0.3:35655/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:37357 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-838v07on | |
Worker: 1
| Comm: tcp://172.17.0.3:41945 | Total threads: 1 |
| Dashboard: http://172.17.0.3:37585/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:44171 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-7f2q0mh_ | |
Worker: 2
| Comm: tcp://172.17.0.3:35431 | Total threads: 1 |
| Dashboard: http://172.17.0.3:33325/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:41919 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-7xngzd99 | |
Worker: 3
| Comm: tcp://172.17.0.3:35923 | Total threads: 1 |
| Dashboard: http://172.17.0.3:42583/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:41215 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-2inukot4 | |
Worker: 4
| Comm: tcp://172.17.0.3:34447 | Total threads: 1 |
| Dashboard: http://172.17.0.3:37235/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:34773 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-_am9g_g_ | |
Worker: 5
| Comm: tcp://172.17.0.3:43755 | Total threads: 1 |
| Dashboard: http://172.17.0.3:38695/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:46303 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-3lr18ilh | |
Worker: 6
| Comm: tcp://172.17.0.3:34329 | Total threads: 1 |
| Dashboard: http://172.17.0.3:34039/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:39797 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-s3hdjjlu | |
Worker: 7
| Comm: tcp://172.17.0.3:36197 | Total threads: 1 |
| Dashboard: http://172.17.0.3:37835/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:41287 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-wh9l3kj9 | |
Worker: 8
| Comm: tcp://172.17.0.3:43743 | Total threads: 1 |
| Dashboard: http://172.17.0.3:43607/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:46681 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-q6oh7t46 | |
Worker: 9
| Comm: tcp://172.17.0.3:44963 | Total threads: 1 |
| Dashboard: http://172.17.0.3:38127/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:40361 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-mxlpf4tm | |
Worker: 10
| Comm: tcp://172.17.0.3:37509 | Total threads: 1 |
| Dashboard: http://172.17.0.3:40533/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:41209 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-xnrrrp3d | |
Worker: 11
| Comm: tcp://172.17.0.3:39001 | Total threads: 1 |
| Dashboard: http://172.17.0.3:36455/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:34753 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-q01e19l0 | |
Worker: 12
| Comm: tcp://172.17.0.3:41381 | Total threads: 1 |
| Dashboard: http://172.17.0.3:41471/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:38633 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-igclkn5h | |
Worker: 13
| Comm: tcp://172.17.0.3:34409 | Total threads: 1 |
| Dashboard: http://172.17.0.3:38969/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:37243 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-78er5h3k | |
Worker: 14
| Comm: tcp://172.17.0.3:36389 | Total threads: 1 |
| Dashboard: http://172.17.0.3:38497/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:44871 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-ltz5em4b | |
Worker: 15
| Comm: tcp://172.17.0.3:36569 | Total threads: 1 |
| Dashboard: http://172.17.0.3:42727/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:37159 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-0lz7_48q | |
Create TIF chips sampled from the mosaic.¶
specs = {
'chip_size': 32,
'bands': [1, 2, 3], #RGB
'mosaic_p': (
'/home/jovyan/work/urbangrammar_samba/'
'ghs_composite_s2/GHS-composite-S2.vrt'
),
'folder': (
'/home/jovyan/work/chips_gb/32/all/'
),
}
for t in types:
df = geopandas.read_parquet(f"/home/jovyan/work/chips_gb/temp/chip_bounds_per_type/{t}.parquet")
centroid = df.centroid
df['X'] = centroid.x.astype(int)
df['Y'] = centroid.y.astype(int)
tools.spilled_bag_of_chips(df, specs, npartitions=16)
print(t)
0_0
1_0
3_0
4_0
5_0
6_0
7_0
8_0
2_0
2_1
2_2
9_0
9_1
9_2
9_4
9_5
Split into train, validation and secret.¶
split = (.7, .15, .15)
for t in types:
for subset in ["train", "validation", "secret"]:
os.makedirs(f"{specs['folder'][:-4]}{subset}/{Path(t).stem}", exist_ok=True)
files = glob.glob(specs["folder"] + t + "/*.tif")
count = len(files)
for f in tqdm(files[:int(count * split[0])]):
f = Path(f)
shutil.copy(f, str(f.parent.parent.parent) + "/train/" + f.parent.stem + "/" + f.name)
for f in tqdm(files[int(count * split[0]):int(count * (split[0] + split[1]))]):
f = Path(f)
shutil.copy(f, str(f.parent.parent.parent) + "/validation/" + f.parent.stem + "/" + f.name)
for f in tqdm(files[int(count * (split[0] + split[1])):]):
f = Path(f)
shutil.copy(f, str(f.parent.parent.parent) + "/secret/" + f.parent.stem + "/" + f.name)
print(t, "done")
0_0 done
1_0 done
3_0 done
4_0 done
5_0 done
6_0 done
7_0 done
8_0 done
2_0 done
2_1 done
2_2 done
9_0 done
9_1 done
9_2 done
9_4 done
9_5 done
client.restart()
Exception in thread AsyncProcess Dask Worker process (from Nanny) watch process join:
Traceback (most recent call last):
File "/opt/conda/lib/python3.9/threading.py", line 973, in _bootstrap_inner
self.run()
File "/opt/conda/lib/python3.9/threading.py", line 910, in run
self._target(*self._args, **self._kwargs)
File "/opt/conda/lib/python3.9/site-packages/distributed/process.py", line 234, in _watch_process
assert exitcode is not None
AssertionError
distributed.nanny - WARNING - Worker process still alive after 1 seconds, killing
Client
Client-b6623fc4-7e03-11ec-af24-f78fed6e375f
| Connection method: Cluster object | Cluster type: distributed.LocalCluster |
| Dashboard: http://127.0.0.1:8787/status |
Cluster Info
LocalCluster
470c4797
| Dashboard: http://127.0.0.1:8787/status | Workers: 16 |
| Total threads: 16 | Total memory: 125.54 GiB |
| Status: running | Using processes: True |
Scheduler Info
Scheduler
Scheduler-43cd5647-3e3e-4ecc-b340-4e08f29fd53b
| Comm: tcp://127.0.0.1:37837 | Workers: 16 |
| Dashboard: http://127.0.0.1:8787/status | Total threads: 16 |
| Started: 8 minutes ago | Total memory: 125.54 GiB |
Workers
Worker: 0
| Comm: tcp://172.17.0.3:38331 | Total threads: 1 |
| Dashboard: http://172.17.0.3:37855/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:43343 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-07kx1avu | |
Worker: 1
| Comm: tcp://172.17.0.3:34243 | Total threads: 1 |
| Dashboard: http://172.17.0.3:37765/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:44491 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-h_kdxwv4 | |
Worker: 2
| Comm: tcp://172.17.0.3:42265 | Total threads: 1 |
| Dashboard: http://172.17.0.3:41089/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:43555 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-_pam4hfo | |
Worker: 3
| Comm: tcp://172.17.0.3:41769 | Total threads: 1 |
| Dashboard: http://172.17.0.3:43029/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:42991 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-nrt11zce | |
Worker: 4
| Comm: tcp://172.17.0.3:39159 | Total threads: 1 |
| Dashboard: http://172.17.0.3:41561/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:45251 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-6ihuw0ih | |
Worker: 5
| Comm: tcp://172.17.0.3:33605 | Total threads: 1 |
| Dashboard: http://172.17.0.3:44313/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:45401 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-a9rk0ubv | |
Worker: 6
| Comm: tcp://172.17.0.3:38909 | Total threads: 1 |
| Dashboard: http://172.17.0.3:42063/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:46455 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-ds9iywfk | |
Worker: 7
| Comm: tcp://172.17.0.3:35487 | Total threads: 1 |
| Dashboard: http://172.17.0.3:33689/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:42927 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-qno150do | |
Worker: 8
| Comm: tcp://172.17.0.3:45403 | Total threads: 1 |
| Dashboard: http://172.17.0.3:40895/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:41521 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-y7v8m4l_ | |
Worker: 9
| Comm: tcp://172.17.0.3:39263 | Total threads: 1 |
| Dashboard: http://172.17.0.3:40831/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:40151 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-i8qavemb | |
Worker: 10
| Comm: tcp://172.17.0.3:42435 | Total threads: 1 |
| Dashboard: http://172.17.0.3:33763/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:41137 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-s3yexqfb | |
Worker: 11
| Comm: tcp://172.17.0.3:42279 | Total threads: 1 |
| Dashboard: http://172.17.0.3:33551/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:41703 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-m5dg5w6t | |
Worker: 12
| Comm: tcp://172.17.0.3:45723 | Total threads: 1 |
| Dashboard: http://172.17.0.3:38281/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:36723 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-nqgkwv39 | |
Worker: 13
| Comm: tcp://172.17.0.3:34501 | Total threads: 1 |
| Dashboard: http://172.17.0.3:40153/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:46727 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-v18po7bt | |
Worker: 14
| Comm: tcp://172.17.0.3:45053 | Total threads: 1 |
| Dashboard: http://172.17.0.3:46347/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:34823 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-svaqd_g1 | |
Worker: 15
| Comm: tcp://172.17.0.3:41303 | Total threads: 1 |
| Dashboard: http://172.17.0.3:40501/status | Memory: 7.85 GiB |
| Nanny: tcp://127.0.0.1:46169 | |
| Local directory: /home/jovyan/work/signature_ai/dask-worker-space/worker-426sen9z | |
bds = polygons_within[~polygons_within["signature_type"].isin(["0_0", "4_0", "7_0"])].compute()
import xyzservices
bds.explore("signature_type",cmap="tab20", prefer_canvas=True,
tiles=xyzservices.TileProvider.from_qms("EOX::Maps - Sentinel-2 cloudless")(max_zoom=22),
max_zoom=22)